package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"bytes"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"
	_ "github.com/go-sql-driver/mysql"
	"database/sql"
)

func GbkToUtf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func getRequest(url string) ([]byte) {
	res, err := stmt1.Query(url)

	if err != nil {
		fmt.Println(err)
	}

	defer res.Close()

	if res.Next() {
		var u, c string
		res.Scan(&u, &c)

		//fmt.Println(c)
		return []byte(c);
	} else {
		fmt.Println(url)
	}


	client := &http.Client{}
	req, err := http.NewRequest("POST", url, nil)
	//req.Header.Set("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
	//req.Header.Set("Accept-Charset","GBK,utf-8;q=0.7,*;q=0.3")
	//req.Header.Set("Accept-Encoding","gzip,deflate,sdch")
	//req.Header.Set("Accept-Language","zh-CN,zh;q=0.8")
	//req.Header.Set("Cache-Control","max-age=0")
	//req.Header.Set("Connection","keep-alive")
	req.Header.Set("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.5048.400 QQBrowser/9.7.13126.400")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("Cookie", "name=anny")
	resp, err := client.Do(req)

	//resp, err := http.Get(url)

	if err != nil {
		fmt.Println(err)
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		fmt.Println(err)
	}

	c, err := GbkToUtf8(body)
	if err != nil {
		fmt.Println(err)
	}

	stmt2.Exec(url, string(c))

	return c
}

func matchChildren(url, pCode string)  {
	exp1 := regexp.MustCompile(`<tr class='(.*?)'>(.*?)<\/tr>`)
	exp2 := regexp.MustCompile(`<td><a href='(.*?)'>(.*?)</a></td><td><a href='(.*?)'>(.*?)</a></td>`)
	exp3 := regexp.MustCompile(`<td>(.*?)</td><td>(.*?)</td>`)
	exp4 := regexp.MustCompile(`<td>(.*)</td><td>(.*)</td><td>(.*)</td>`)
	exp5 := regexp.MustCompile(`\d+?.html`)
	body := getRequest(url)

	result := exp1.FindAllStringSubmatch(string(body), -1)
	for _, list := range result {
		if string(list[1]) == `citytr` {
			res := exp2.FindStringSubmatch(string(list[2]))
			if len(res) == 0 {
				fmt.Println(list[2])
				fmt.Println("exp2")
			} else {
				matchChildren(exp5.ReplaceAllString(url, res[1]), res[2])
				_, err := stmt.Exec(res[2], "", res[4], pCode, string(list[1]))
				if err!=nil {
					fmt.Println(err)
					fmt.Println(res[2], "", res[4], pCode)
					fmt.Println(url)
					fmt.Println("*1")
				}
			}
			//fmt.Println(res)
		} else if string(list[1]) == `countytr` {
			res := exp2.FindStringSubmatch(string(list[2]))

			if len(res) == 0 {
				res = exp3.FindStringSubmatch(string(list[2]))
				if len(res) == 0 {
					fmt.Println(list[2])
				} else {
					_, err := stmt.Exec(res[1], "", res[2], pCode, string(list[1]))
					if err!=nil {
						fmt.Println(err)
						fmt.Println(res[1], "", res[2], pCode)
						fmt.Println(url)
						fmt.Println("*2")
					}
				}
			} else {
				matchChildren(exp5.ReplaceAllString(url, res[1]), res[2])
				_, err := stmt.Exec(res[2], "", res[4], pCode, string(list[1]))
				if err!=nil {
					fmt.Println(err)
					fmt.Println(res[2], "", res[4], pCode)
					fmt.Println(url)
					fmt.Println("*3")
				}
			}
		} else if string(list[1]) == `towntr` {
			res := exp2.FindStringSubmatch(string(list[2]))

			if len(res) == 0 {
				fmt.Println(list[2])
			} else {
				matchChildren(exp5.ReplaceAllString(url, res[1]), res[2])
				_, err := stmt.Exec(res[2], "", res[4], pCode, string(list[1]))
				if err!=nil {
					fmt.Println(err)
					fmt.Println(res[2], "", res[4], pCode)
					fmt.Println(url)
					fmt.Println("*4")
				}
			}
		} else if list[1] == `villagetr`{
			res := exp4.FindStringSubmatch(string(list[2]))

			if len(res) == 0 {
				fmt.Println(list[2])
				fmt.Println("**")
			} else {
				_, err := stmt.Exec(res[1], res[2], res[3], pCode, string(list[1]))
				if err!=nil {
					fmt.Println(err)
					fmt.Println(res[1], res[2], res[3], pCode)
					fmt.Println(url)
					fmt.Println("*5")
				}
			}
		} else if list[1] == `villagehead` {
		} else {
			fmt.Println(list)
		}
	}
}
const (
	TIMES = 30
)
var stmt, stmt1, stmt2 *sql.Stmt

func main() {
	db, _ := sql.Open("mysql", "root:@tcp(127.0.0.1:3306)/test?charset=utf8")
	stmt, _ = db.Prepare("INSERT xzqh SET code=?,cx=?,name=?,pCode=?,class=?")
	stmt1, _ = db.Prepare("SELECT * FROM pages WHERE url=?")
	stmt2, _ = db.Prepare("INSERT pages SET url=?,page=?")

	exp1 := regexp.MustCompile(`<a href='(.*?).html'>(.*?)<br\/><\/a>`)
	body := getRequest("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/index.html")
	result1 := exp1.FindAllStringSubmatch(string(body), -1)
	for _, text := range result1 {

		code := text[1]
		_, err := stmt.Exec(code + "0000000000", "", text[2], "", "provincetr")
		if err!=nil {
			fmt.Println(err)
			fmt.Println("main")
		}
		go matchChildren("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/" + code + ".html", code + "0000000000")
	}

	if (len(result1) == 0) {
		fmt.Println("errorxxxxx")
	}
	var str string
	fmt.Scan(&str)
}
